package org.fujene.memcon;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Logger;

import org.fujene.constants.Constants;
import org.fujene.diskblock.DiskBlock;
import org.fujene.diskblock.AppendixDiskBlock;
import org.fujene.diskblock.SkipPointerDiskBlock;
import org.fujene.diskblock.blockwise.BlockwiseContentDiskBlock;
import org.fujene.diskblock.blockwise.BlockwiseDictionaryDiskBlock;
import org.fujene.diskblock.blockwise.BlockwiseInvertIndexDiskBlock;
import org.fujene.memman.BlockwiseMemoryManager;
import org.fujene.memoryblock.MemoryBlock;
import org.fujene.memoryblock.AppendixBlock;
import org.fujene.memoryblock.SkipPointerBlock;
import org.fujene.memoryblock.blockwise.BlockwiseContentBlock;
import org.fujene.memoryblock.blockwise.BlockwiseDictionaryBlock;
import org.fujene.memoryblock.blockwise.BlockwiseInvertIndexBlock;
import org.fujene.structure.ContentInfo;
import org.fujene.structure.ExtendTerm;
import org.fujene.structure.LookupPointer;
import org.fujene.structure.exception.DataMatchingException;
import org.fujene.toolbox.ByteConverter;

public class BlockwiseIndexMemory{
	private static final int MEM_TYPES = Constants.MEMORY_TYPE_KINDS; // そのうち拡張する予定
	private static final int MEM_DICT = Constants.MEMORY_TYPE_DICT;
	private static final int MEM_INVINDEX = Constants.MEMORY_TYPE_INVINDEX;
	private static final int MEM_SKIPPTR = Constants.MEMORY_TYPE_SKIPPTR;
	private static final int MEM_CONT = Constants.MEMORY_TYPE_CONTENTS;
	private static final int MEM_PURECONT = Constants.MEMORY_TYPE_APPENDIX;
	private static final int DEFAULT_OPT_LEN = 1;

	// クラスメンバ ブロックをなんとか表す

	// ブロック本体
	// 各ブロックのサイズ
	private int blockSize = 1048576; // 1MB
	// ブロック数
	private int blockNum = 256;

	private MemoryBlock[] blocks = null;
	private int[] writeBackIDs = null;
	private int nextID = 0;

	private int[] firstBlock = null;
	private int[] lastBlock = null;

	private int contentNum;
	private int supplementNum;
	private int[] contentFirst;
	private int[] contentLast;
	private int[] supplementFirst;
	private int[] supplementLast;
	private BitSet isLocal;
	private ArrayList<Integer> blockTypes;

	// ディスクとの連絡用
	private IndexDisk disk = null;
	private String dir = null;
	private BlockwiseMemoryManager memman = null;
	private ContentInfo contentInfo = null;
	private Logger logger = null;

	private ExecutorService execsrv = null;

	public BlockwiseIndexMemory(int blockSize, int blockNum, IndexDisk disk, ContentInfo contentInfo,
			Logger logger, BlockwiseMemoryManager memman){
		setBlockSize(blockSize);
		setBlockNum(blockNum);
		this.disk = disk;
		this.dir = disk.getDir();
		this.contentNum = contentInfo.getContentNum();
		this.contentFirst = new int[this.contentNum];
		this.contentLast = new int[this.contentNum];
		this.supplementNum = contentInfo.getAppendixNum();
		this.supplementFirst = new int[this.supplementNum];
		this.supplementLast = new int[this.supplementNum];
		this.contentInfo = contentInfo;
		this.logger = logger;
		this.memman = memman;
		this.blockTypes = new ArrayList<Integer>();
		this.execsrv = Executors.newFixedThreadPool(4);

		initBlock();
	}

	// 一番最初に使おう
	private void initBlock(){
		this.blocks = new MemoryBlock[this.blockNum];
		this.writeBackIDs = new int[MEM_TYPES];

		this.firstBlock = new int[MEM_TYPES];
		this.lastBlock = new int[MEM_TYPES];

		for(int i = 0; i < MEM_TYPES; i++){
			this.firstBlock[i] = -1;
			this.lastBlock[i] = -1;
			this.writeBackIDs[i] = -1;
		}

		for(int i = 0; i < this.contentNum; i++){
			this.contentFirst[i] = -1;
			this.contentLast[i] = -1;
		}

		for(int i = 0; i < this.supplementNum; i++){
			this.supplementFirst[i] = -1;
			this.supplementLast[i] = -1;
		}

		isLocal = new BitSet(128);

		initializeAllocate();
	}

	// 初期アロケート 種類追加時はここをいじる
	private void initializeAllocate(){
		int i;
		allocateBlock(nextID++, MEM_DICT, 0);
		allocateBlock(nextID++, MEM_SKIPPTR, 0);
		allocateBlock(nextID++, MEM_INVINDEX, 0);
		for(i = 0; i < contentNum; i++)
			allocateBlock(nextID++, MEM_CONT, i);
		for(i = 0; i < supplementNum; i++)
			allocateBlock(nextID++, MEM_PURECONT, i);
	}

	private int allocateBlock(int index, int type, int contentType){
		if(memman.storeLocally(index)){
			for(int i = 0; i < blocks.length; i++){
				if(i == blocks.length - 1 && blocks[i] != null){
					int mover = -1;
					// メモリの空きがなかったのでディスクに寄せる
					// 優先順: PureContents -> Contents -> Inv. Index ->
					// (→はしばらく凍結)Dictionary ->
					// Skip ptr.
					if(getMemoryAmount(MEM_PURECONT) > 2){
						mover = getMinimumBlock(MEM_PURECONT);
						saveMemory(mover);
						i = mover;
					}else if(getMemoryAmount(MEM_CONT) > 2){
						mover = getMinimumBlock(MEM_CONT);
						saveMemory(mover);
						i = mover;
					}else if(getMemoryAmount(MEM_INVINDEX) > 2){
						mover = getMinimumBlock(MEM_INVINDEX);
						saveMemory(mover);
						i = mover;
						// } else if (getMemoryAmount(MEM_DICT) > 2) {
						// mover = getMinimumBlock(MEM_DICT);
						// saveMemory(mover);
						// i = mover;
						// } else if (getMemoryAmount(MEM_SKIPPTR) > 2) {
						// mover = getMinimumBlock(MEM_SKIPPTR);
						// saveMemory(mover);
						// i = mover;
					}
				}

				if(blocks[i] == null){
					switch(type){
					case MEM_DICT:
						blocks[i] = new BlockwiseDictionaryBlock(blockSize, index, true);
						if(firstBlock[MEM_DICT] == -1) firstBlock[MEM_DICT] = index;
						lastBlock[MEM_DICT] = index;
						blocks[i].setOption(1, contentNum);
						break;
					case MEM_SKIPPTR:
						blocks[i] = new SkipPointerBlock(blockSize, index, true);
						if(firstBlock[MEM_SKIPPTR] == -1) firstBlock[MEM_SKIPPTR] = index;
						else blocks[lastBlock[MEM_SKIPPTR]].setOption(1, index);
						lastBlock[MEM_SKIPPTR] = index;
						blocks[i].setOption(1, -1);
						blocks[i].setOption(2, MEM_SKIPPTR);
						break;
					case MEM_INVINDEX:
						blocks[i] = new BlockwiseInvertIndexBlock(blockSize, index, true);
						if(firstBlock[MEM_INVINDEX] == -1) firstBlock[MEM_INVINDEX] = index;
						lastBlock[MEM_INVINDEX] = index;
						break;
					case MEM_CONT:
						blocks[i] = new BlockwiseContentBlock(blockSize, index, true);
						if(contentFirst[contentType] == -1) contentFirst[contentType] = index;
						contentLast[contentType] = index;
						blocks[i].setOption(1, contentInfo.getConnectionNum(contentType));
						blocks[i].setOption(2, contentType);
						break;
					case MEM_PURECONT:
						blocks[i] = new AppendixBlock(blockSize, index, true);
						if(supplementFirst[contentType] == -1) supplementFirst[contentType] = index;
						supplementLast[contentType] = index;
						blocks[i].setOption(0, contentType);
						break;
					default:
						// タイプ外
						throw new IndexOutOfBoundsException();
					}
					isLocal.set(index);
					memman.setBlock(index, type, contentType);
					if(index < blockTypes.size()) blockTypes.set(index,
							Integer.valueOf(contentType * 100 + type));
					else blockTypes.add(Integer.valueOf(contentType * 100 + type));

					return i;
				}
			}
			return -1;
		}else return index;
	}

	private int getMemoryAmount(int type){
		int amount = 0;

		for(int i = 0; i < blocks.length; i++){
			if(blocks[i] != null && blocks[i].getType() == type) amount++;
		}

		return amount;
	}

	// ローカル限定
	private int getBlockArrayIndex(int ID){
		if(!isLocal.get(ID)) return -1;
		for(int i = 0; i < blocks.length; i++){
			if(blocks[i] == null) continue;
			if(blocks[i].getID() == ID) return i;
		}

		// なかった
		return -1;
	}

	// グローバル探索
	public ExtendTerm lookupDictionary(LookupPointer ptr){
		if(ptr == null || ptr.equals(LookupPointer.EMPTY)) return null;
		byte[] retbytes;
		Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupDictionary", ptr.block, ptr.toBytes(),
				false, true));
		if(isLocal.get(ptr.block)){
			return lookupDictionaryImpl(ptr);
		}else{
			// 開く
			try{
				retbytes = futureb.get();
				return new ExtendTerm(retbytes);
			}catch(InterruptedException e){
				e.printStackTrace();
			}catch(ExecutionException e){
				e.printStackTrace();
			}
		}
		return null;
	}

	public ExtendTerm lookupDictionaryImpl(LookupPointer ptr){
		if(ptr == null || ptr.equals(LookupPointer.EMPTY)) return null;
		int arrayIndex = getBlockArrayIndex(ptr.block);
		if(arrayIndex != -1){
			return ((BlockwiseDictionaryBlock)blocks[arrayIndex]).getTerm(ptr.point);
		}else{
			// ローカル -> ディスク探索
			if(disk.isExist(ptr.block)){
				try{
					BlockwiseDictionaryDiskBlock dicDisk = new BlockwiseDictionaryDiskBlock(disk.getFileMap()
							.get(Integer.valueOf(ptr.block)));
					return dicDisk.getTerm(ptr.point);
				}catch(Exception e){}
			}
			// ローカルにないと判断
		}

		return null;
	}

	public LookupPointer lookupDictionaryNext(LookupPointer ptr){
		byte[] retbytes;
		Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupDictionaryNext", ptr.block, ptr
				.toBytes(), false, true));
		if(isLocal.get(ptr.block)){
			return lookupDictionaryNextImpl(ptr);
		}else{
			// 開く
			try{
				retbytes = futureb.get();
				if(retbytes != null){
					return LookupPointer.fromBytes(retbytes);
				}else return null;
			}catch(InterruptedException e){}catch(ExecutionException e){}
		}
		return null;
	}

	// 辞書ポインタだけ読み込み
	public LookupPointer lookupDictionaryNextImpl(LookupPointer ptr){
		int arrayIndex = getBlockArrayIndex(ptr.block);

		if(arrayIndex != -1){
			return ((BlockwiseDictionaryBlock)blocks[arrayIndex]).getNextPointer(ptr.point);
		}else{
			if(disk.isExist(ptr.block)){
				try{
					BlockwiseDictionaryDiskBlock dicDisk = new BlockwiseDictionaryDiskBlock(disk.getFileMap()
							.get(Integer.valueOf(ptr.block)));
					return dicDisk.getNextPointer(ptr.point);
				}catch(Exception e){}
			}
		}

		return null;
	}

	public LookupPointer lookupDictionaryInvIndex(LookupPointer ptr, int index){
		if(ptr == null || ptr.equals(LookupPointer.EMPTY)) return null;

		try{
			ByteArrayOutputStream stream = new ByteArrayOutputStream();
			stream.write(ptr.toBytes());
			stream.write(ByteConverter.intToByte(index, 4));
			Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupDictionaryInvIndex", ptr.block,
					stream.toByteArray(), false, true));
			if(isLocal.get(ptr.block)){
				return lookupDictionaryInvIndexImpl(ptr, index);
			}else{
				// 開く
				return LookupPointer.fromBytes(futureb.get());
			}
		}catch(IOException e){}catch(InterruptedException e){}catch(ExecutionException e){}
		return null;
	}

	// 転置インデックスポインタだけ読み込み
	public LookupPointer lookupDictionaryInvIndexImpl(LookupPointer ptr, int index){
		int arrayIndex = getBlockArrayIndex(ptr.block);

		if(arrayIndex != -1){
			return ((BlockwiseDictionaryBlock)blocks[arrayIndex]).getDocIDPtr(ptr.point, index);
		}else{
			if(disk.isExist(ptr.block)){
				try{
					BlockwiseDictionaryDiskBlock dicDisk = new BlockwiseDictionaryDiskBlock(disk.getFileMap()
							.get(Integer.valueOf(ptr.block)));
					return dicDisk.getDocIDPointer(ptr.point, index);
				}catch(Exception e){}
			}
		}

		return null;
	}

	public LookupPointer storeDictionary(String str, LookupPointer next) throws UnsupportedEncodingException{
		try{
			ByteArrayOutputStream stream = new ByteArrayOutputStream();
			if(next != null) stream.write(next.toBytes());
			else stream.write(LookupPointer.EMPTY.toBytes());
			stream.write(str.getBytes("UTF-8"));
			Future<byte[]> futureb = execsrv.submit(new FuncCaller("storeDictionary", lastBlock[MEM_DICT],
					stream.toByteArray(), false, false));
			if(isLocal.get(lastBlock[MEM_DICT])){
				return storeDictionaryImpl(str, next);
			}else{
				// 開く
				return LookupPointer.fromBytes(futureb.get());
			}
		}catch(IOException e){}catch(InterruptedException e){}catch(ExecutionException e){}

		return null;
	}

	public LookupPointer storeDictionaryImpl(String str, LookupPointer next)
			throws UnsupportedEncodingException{
		int dictLast = getBlockArrayIndex(lastBlock[MEM_DICT]);

		// System.out.println("Set next: " + next);

		BlockwiseDictionaryBlock dicBlock = (BlockwiseDictionaryBlock)blocks[dictLast];
		// 置き場探し
		int newPtr = dicBlock.getNewPointer(str);
		if(newPtr == -1){
			// 他をあたる
			if(memman.existsLocal(nextID)){
				int nextBlock = allocateBlock(nextID++, MEM_DICT, 0);
				if(nextBlock != -1){
					dictLast = nextBlock;
					dicBlock = (BlockwiseDictionaryBlock)blocks[nextBlock];
					newPtr = dicBlock.getNewPointer(str);
				}
			}else{
				// どこかにブロック作成
			}
		}

		dicBlock.setTerm(newPtr, next, str);
		writeBackIDs[MEM_DICT]++;
		return new LookupPointer(dicBlock.getID(), newPtr);
	}

	public void storeDictionaryPointer(LookupPointer dicPtr, LookupPointer next){
		try{
			ByteArrayOutputStream stream = new ByteArrayOutputStream();
			stream.write(dicPtr.toBytes());
			stream.write(next.toBytes());
			execsrv.submit(new FuncCaller("storeDictionaryPointer", dicPtr.block, stream.toByteArray(),
					false, false));
			if(isLocal.get(dicPtr.block)){
				storeDictionaryPointerImpl(dicPtr, next);
			}else{}
		}catch(IOException e){}
	}

	// ※追加しない
	public void storeDictionaryPointerImpl(LookupPointer dicPtr, LookupPointer next){
		int arrayIndex = getBlockArrayIndex(dicPtr.block);

		if(arrayIndex != -1){
			try{
				((BlockwiseDictionaryBlock)blocks[arrayIndex]).setNextPointer(dicPtr.point, next);
			}catch(Exception e){}
		}else{
			// ファイル書き換え対応はのちほど
		}
	}

	// ※上書き
	public void storeDictionaryToInvIndex(LookupPointer dicPtr, LookupPointer invIndex, int index,
			boolean isIncrement){
		try{
			ByteArrayOutputStream stream = new ByteArrayOutputStream();
			stream.write(dicPtr.toBytes());
			stream.write(invIndex.toBytes());
			stream.write(ByteConverter.intToByte(index, 3));
			stream.write(isIncrement ? 1 : 0);
			execsrv.submit(new FuncCaller("storeDictionaryToInvIndex", dicPtr.block, stream.toByteArray(),
					false, false));
			if(isLocal.get(dicPtr.block)){
				storeDictionaryToInvIndexImpl(dicPtr, invIndex, index, isIncrement);
			}else{}
		}catch(IOException e){}
	}

	public void storeDictionaryToInvIndexImpl(LookupPointer dicPtr, LookupPointer invIndex, int index,
			boolean isIncrement){
		int arrayIndex = getBlockArrayIndex(dicPtr.block);

		if(arrayIndex != -1){
			try{
				((BlockwiseDictionaryBlock)blocks[arrayIndex]).setDocIDptr(dicPtr.point, index, invIndex,
						isIncrement);
			}catch(Exception e){}
		}else{
			// ファイル書き換え対応はのちほど
		}
	}

	public LookupPointer lookupSkipPointer(int index){
		byte[] retb;
		Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupSkipPointer", firstBlock[MEM_SKIPPTR],
				ByteConverter.intToByte(index, 4), false, true));
		if(isLocal.get(firstBlock[MEM_SKIPPTR])){
			return lookupSkipPointerImpl(index);
		}else{
			try{
				retb = futureb.get();
				if(retb != null) return LookupPointer.fromBytes(retb);
				else return null;
			}catch(InterruptedException e){}catch(ExecutionException e){}
		}
		return null;
	}

	public LookupPointer lookupSkipPointerImpl(int index){
		int nextBlockNum = getBlockArrayIndex(firstBlock[MEM_SKIPPTR]);
		SkipPointerBlock skipBlock = null;
		int blockIndex = getBlockArrayIndex(nextBlockNum);
		int findLength = 0;
		int length = 0;

		// ブロック探索
		skipBlock = (SkipPointerBlock)blocks[blockIndex];
		// スキップポインタなし
		if(skipBlock.getLength() == 0) return null;

		// インデックス探し
		do{
			length = skipBlock.getLength();
			findLength += length;
			if(findLength >= index) break;
			nextBlockNum = skipBlock.getNextblock();
			blockIndex = getBlockArrayIndex(nextBlockNum);
			skipBlock = (SkipPointerBlock)blocks[blockIndex];
		}while(nextBlockNum != -1);

		// そんなになかった
		if(findLength < index || skipBlock == null) return null;

		index -= (findLength - length);
		return skipBlock.getPointer(index);
	}

	public LookupPointer[] lookupContentPurePointer(LookupPointer ptr){
		byte[] retbytes;
		Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupContentPurePointer", ptr.block, ptr
				.toBytes(), false, true));
		if(isLocal.get(ptr.block)){
			return lookupContentPurePointerImpl(ptr);
		}else{
			try{
				retbytes = futureb.get();

				// 開く
				LookupPointer[] ptrs = new LookupPointer[ByteConverter.byteToInt(retbytes, 0, 2)];
				for(int i = 0; i < ptrs.length; i++){
					ptrs[i] = LookupPointer.fromBytes(retbytes, 2 + (6 * i));
				}
				return ptrs;
			}catch(InterruptedException e){}catch(ExecutionException e){}
		}
		return null;
	}

	public LookupPointer[] lookupContentPurePointerImpl(LookupPointer ptr){
		int arrayIndex = getBlockArrayIndex(ptr.block);

		if(arrayIndex != -1){
			return ((BlockwiseContentBlock)blocks[arrayIndex]).getSuppPtr(ptr.point);
		}else{
			if(disk.isExist(ptr.block)){
				try{
					File desc = disk.getDescriptor(ptr.block);
					BlockwiseContentDiskBlock contentDisk = new BlockwiseContentDiskBlock(ptr.block, desc,
							blockSize, DEFAULT_OPT_LEN);
					return contentDisk.getSuppPtr(ptr.point);
				}catch(Exception e){}
			}
		}

		return null;
	}

	// 単語スキップポインタのみ
	public int getSkipPointerLength(){
		Future<byte[]> futureb = execsrv.submit(new FuncCaller("getSkipPointerLength",
				firstBlock[MEM_SKIPPTR], new byte[0], true, true));
		if(isLocal.get(firstBlock[MEM_SKIPPTR])){
			return getSkipPointerLengthImpl();
		}else{
			try{
				return ByteConverter.byteToInt(futureb.get());
			}catch(InterruptedException e){}catch(ExecutionException e){}
		}
		return -1;
	}

	public int getSkipPointerLengthImpl(){
		int nextBlockNum = getBlockArrayIndex(firstBlock[MEM_SKIPPTR]);
		SkipPointerBlock skipBlock = null;
		int blockIndex = -1;
		int findIndex = 0;

		// ブロック探索
		do{
			blockIndex = getBlockArrayIndex(nextBlockNum);
			skipBlock = (SkipPointerBlock)blocks[blockIndex];
			int length = skipBlock.getLength();
			findIndex += length;
			nextBlockNum = skipBlock.getNextblock();
		}while(nextBlockNum != -1);

		return findIndex;
	}

	// 本文スキップポインタ対応
	public void storeSkipPointer(int index, LookupPointer ptr, boolean isModify){
		try{
			ByteArrayOutputStream stream = new ByteArrayOutputStream();
			stream.write(ByteConverter.intToByte(index, 3));
			stream.write(ptr.toBytes());
			stream.write(ByteConverter.intToByte((isModify ? 1 : 0), 1));
			execsrv.submit(new FuncCaller("storeSkipPointer", firstBlock[MEM_SKIPPTR], stream.toByteArray(),
					false, false));
			if(isLocal.get(firstBlock[MEM_SKIPPTR])) storeSkipPointerImpl(index, ptr, isModify);
		}catch(IOException e){}
	}

	public void storeSkipPointerImpl(int index, LookupPointer ptr, boolean isModify){
		int nextBlockNum = getBlockArrayIndex(firstBlock[MEM_SKIPPTR]);
		int blockIndex = -1;
		int findIndex = 0;
		SkipPointerBlock skipBlock = null;
		SkipPointerBlock setBlock = null;

		// 置き場探し
		do{
			blockIndex = getBlockArrayIndex(nextBlockNum);
			skipBlock = (SkipPointerBlock)blocks[blockIndex];
			int length = skipBlock.getLength();
			findIndex += length;
			if(findIndex - index >= 0){
				setBlock = skipBlock;
				break;
			}
			nextBlockNum = skipBlock.getNextblock();
		}while(nextBlockNum != -1);

		if(setBlock != null){
			if(isModify) setBlock.setPointer(index, ptr, isModify);
			else{
				byte[] moveVal = null;
				moveVal = setBlock.setPointer(index, ptr, isModify);
				if(moveVal != null){
					do{
						nextBlockNum = skipBlock.getNextblock();
						blockIndex = getBlockArrayIndex(nextBlockNum);
						setBlock = (SkipPointerBlock)blocks[blockIndex];
						moveVal = setBlock.movePointerRep(moveVal);
					}while(moveVal != null);
				}
			}
		}

		writeBackIDs[MEM_SKIPPTR]++;
	}

	// デバッグ用 スキップポインタ一覧を出力
	// 本文スキップポインタ対応
	public void showSkipPointer(int type){
		int nextBlockNum = getBlockArrayIndex(firstBlock[type]);
		SkipPointerBlock skipBlock = null;
		int blockIndex = -1;
		LookupPointer output = null;

		// ブロック探索
		do{
			blockIndex = getBlockArrayIndex(nextBlockNum);
			skipBlock = (SkipPointerBlock)blocks[blockIndex];
			int length = skipBlock.getLength();
			for(int i = 0; i < length; i++){
				output = skipBlock.getPointer(i);
				int dicBlock = getBlockArrayIndex(output.block);
				ExtendTerm newTerm = ((BlockwiseDictionaryBlock)blocks[dicBlock]).getTerm(output.point);
				newTerm.skipptr_index = i;
				logger.fine("Term at skip pointer: " + newTerm.toString());
			}
			nextBlockNum = skipBlock.getNextblock();
		}while(nextBlockNum != -1);
	}

	public LookupPointer lookupNextInvIndex(LookupPointer ptr){
		if(ptr == null || ptr.equals(LookupPointer.EMPTY)) return null;
		Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupNextInvIndex", ptr.block,
				ptr.toBytes(), false, true));
		if(isLocal.get(ptr.block)){
			return lookupNextInvIndexImpl(ptr);
		}else{
			// 開く
			try{
				return LookupPointer.fromBytes(futureb.get());
			}catch(InterruptedException e){}catch(ExecutionException e){}
		}
		return null;
	}

	public LookupPointer lookupNextInvIndexImpl(LookupPointer ptr){
		int arrayIndex = getBlockArrayIndex(ptr.block);

		if(arrayIndex != -1){
			return ((BlockwiseInvertIndexBlock)blocks[arrayIndex]).getInvertIndex(ptr.point).getPrevPtr();
		}else{
			if(disk.isExist(ptr.block)){
				try{
					File desc = disk.getDescriptor(ptr.block);
					BlockwiseInvertIndexDiskBlock invDisk = new BlockwiseInvertIndexDiskBlock(ptr.block,
							desc, blockSize, DEFAULT_OPT_LEN);
					return invDisk.getInvertIndex(ptr.point).getPrevPtr();
				}catch(Exception e){}
			}
		}

		return null;
	}

	public LookupPointer lookupInvIndexContentPtr(LookupPointer ptr){
		if(ptr == null || ptr.equals(LookupPointer.EMPTY)) return null;
		Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupInvIndexContentPtr", ptr.block, ptr
				.toBytes(), false, true));
		if(isLocal.get(ptr.block)){
			return lookupInvIndexContentPtrImpl(ptr);
		}else{
			// 開く
			try{
				return LookupPointer.fromBytesLong(futureb.get());
			}catch(InterruptedException e){
				e.printStackTrace();
			}catch(ExecutionException e){
				e.printStackTrace();
			}
		}
		return null;
	}

	public LookupPointer lookupInvIndexContentPtrImpl(LookupPointer ptr){
		int arrayIndex = getBlockArrayIndex(ptr.block);

		if(arrayIndex != -1){
			return ((BlockwiseInvertIndexBlock)blocks[arrayIndex]).getInvertIndex(ptr.point).getContentPtr();
		}else{
			if(disk.isExist(ptr.block)){
				try{
					File desc = disk.getDescriptor(ptr.block);
					BlockwiseInvertIndexDiskBlock invDisk = new BlockwiseInvertIndexDiskBlock(ptr.block,
							desc, blockSize, DEFAULT_OPT_LEN);
					return invDisk.getInvertIndex(ptr.point).getContentPtr();
				}catch(Exception e){}
			}
		}

		return null;
	}

	public LookupPointer storeInvertIndex(LookupPointer prev, LookupPointer contentPtr, int termfreq){
		try{
			ByteArrayOutputStream stream = new ByteArrayOutputStream();
			stream.write(prev.toBytes());
			stream.write(contentPtr.toBytes());
			stream.write(ByteConverter.intToByte(termfreq, 2));
			Future<byte[]> futureb = execsrv.submit(new FuncCaller("storeInvertIndex",
					lastBlock[MEM_INVINDEX], stream.toByteArray(), false, false));
			if(isLocal.get(lastBlock[MEM_INVINDEX])){
				return storeInvertIndexImpl(prev, contentPtr, termfreq);
			}else{
				return LookupPointer.fromBytes(futureb.get());
			}
		}catch(IOException e){}catch(InterruptedException e){}catch(ExecutionException e){}

		return null;
	}

	public LookupPointer storeInvertIndexImpl(LookupPointer prev, LookupPointer contentPtr, int termfreq){
		int invertLast = getBlockArrayIndex(lastBlock[MEM_INVINDEX]);

		BlockwiseInvertIndexBlock invBlock = (BlockwiseInvertIndexBlock)blocks[invertLast];
		// 置き場探し
		int newPtr = invBlock.getLeft();
		if(newPtr == 0){
			// 他をあたる
			int nextBlock = allocateBlock(nextID++, MEM_INVINDEX, 0);
			if(nextBlock != -1){
				invertLast = nextBlock;
				invBlock = (BlockwiseInvertIndexBlock)blocks[nextBlock];
				newPtr = invBlock.getLeft();
			}
		}
		writeBackIDs[MEM_INVINDEX]++;

		return invBlock.setInvertIndex(prev, contentPtr, termfreq);
	}

	public String lookupContents(LookupPointer ptr){
		try{
			Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupContents", ptr.block,
					ptr.toBytes(), false, true));

			if(isLocal.get(ptr.block)){
				return lookupContentsImpl(ptr);
			}else{
				return new String(futureb.get(), "UTF-8");
			}
		}catch(IOException e){}catch(InterruptedException e){}catch(ExecutionException e){}

		return null;
	}

	public String lookupContentsImpl(LookupPointer ptr){
		int arrayIndex = getBlockArrayIndex(ptr.block);

		if(arrayIndex != -1){
			return ((BlockwiseContentBlock)blocks[arrayIndex]).getContent(ptr.point);
		}else{
			if(disk.isExist(ptr.block)){
				try{
					File fileDesc = disk.getDescriptor(ptr.block);
					BlockwiseContentDiskBlock contDisk = new BlockwiseContentDiskBlock(ptr.block, fileDesc,
							blockSize, 2);
					return contDisk.getContent(ptr.point);
				}catch(Exception e){}
			}
		}

		return null;
	}

	public LookupPointer storeContents(String content, LookupPointer[] suppPtrs, long ID, int index){
		try{
			ByteArrayOutputStream stream = new ByteArrayOutputStream();
			stream.write(ByteConverter.intToByte(index, 4));
			stream.write(ByteConverter.longToByte(ID, 8));
			if(suppPtrs != null){
				stream.write(ByteConverter.intToByte(suppPtrs.length, 2));
				for(int i = 0; i < suppPtrs.length; i++){
					stream.write(suppPtrs[i].toBytes());
				}
			}else stream.write(ByteConverter.intToByte(0, 2));
			stream.write(content.getBytes("UTF-8"));
			Future<byte[]> futureb = execsrv.submit(new FuncCaller("storeContents", contentLast[index],
					stream.toByteArray(), false, false));
			if(isLocal.get(contentLast[index])){
				return storeContentsImpl(content, suppPtrs, ID, index);
			}else{
				return LookupPointer.fromBytes(futureb.get());
			}
		}catch(IOException e){}catch(InterruptedException e){}catch(ExecutionException e){}
		return null;
	}

	public LookupPointer storeContentsImpl(String content, LookupPointer[] suppPtrs, long ID, int index){
		int lastContent = getBlockArrayIndex(contentLast[index]);

		BlockwiseContentBlock contBlock = (BlockwiseContentBlock)blocks[lastContent];
		// 置き場探し
		int newPtr = contBlock.getNewPointer(content);
		if(newPtr == -1){
			// 他をあたる
			int nextBlock = allocateBlock(nextID++, MEM_CONT, index);
			if(nextBlock != -1){
				lastContent = nextBlock;
				contBlock = (BlockwiseContentBlock)blocks[nextBlock];
				newPtr = contBlock.getNewPointer(content);
			}
		}

		contBlock.setContent(newPtr, suppPtrs, ID, content);
		writeBackIDs[MEM_CONT]++;

		return new LookupPointer(contBlock.getID(), newPtr);
	}

	public String lookupPureContent(LookupPointer ptr){
		try{
			Future<byte[]> futureb = execsrv.submit(new FuncCaller("lookupPureContent", ptr.block, ptr
					.toBytes(), false, true));
			if(isLocal.get(ptr.block)){
				return lookupContentsImpl(ptr);
			}else{
				return new String(futureb.get(), "UTF-8");
			}
		}catch(IOException e){}catch(InterruptedException e){}catch(ExecutionException e){}

		return null;
	}

	public String lookupPureContentImpl(LookupPointer ptr){
		int arrayIndex = getBlockArrayIndex(ptr.block);

		if(arrayIndex != -1){
			return ((AppendixBlock)blocks[arrayIndex]).getContent(ptr.point);
		}else{
			if(disk.isExist(ptr.block)){
				try{
					File fileDesc = disk.getDescriptor(ptr.block);
					AppendixDiskBlock contDisk = new AppendixDiskBlock(ptr.block, fileDesc, blockSize,
							DEFAULT_OPT_LEN);
					return contDisk.getContent(ptr.point);
				}catch(Exception e){}
			}
		}

		return null;
	}

	public LookupPointer storePureContent(String str, int index){
		try{
			ByteArrayOutputStream stream = new ByteArrayOutputStream();
			stream.write(ByteConverter.intToByte(index, 4));
			stream.write(str.getBytes("UTF-8"));
			Future<byte[]> futureb = execsrv.submit(new FuncCaller("storePureContent", contentLast[index],
					stream.toByteArray(), false, false));
			if(isLocal.get(contentLast[index])){
				return storePureContentImpl(str, index);
			}else{
				return LookupPointer.fromBytes(futureb.get());
			}
		}catch(IOException e){}catch(InterruptedException e){}catch(ExecutionException e){}
		return null;
	}

	public LookupPointer storePureContentImpl(String str, int index){
		int contentLast = getBlockArrayIndex(supplementLast[index]);

		AppendixBlock contBlock = (AppendixBlock)blocks[contentLast];
		// 置き場探し
		int newPtr = contBlock.getNewPointer(str);
		if(newPtr == -1){
			// 他をあたる
			int nextBlock = allocateBlock(nextID++, MEM_PURECONT, 0);
			if(nextBlock != -1){
				contentLast = nextBlock;
				contBlock = (AppendixBlock)blocks[nextBlock];
				newPtr = contBlock.getNewPointer(str);
			}
		}

		contBlock.setContent(newPtr, str);
		writeBackIDs[MEM_PURECONT]++;

		return new LookupPointer(contBlock.getID(), newPtr);
	}

	public void readMemory() throws Exception{
		// 記録を読み出し
		// コピーコンストラクタ
		SortedMap<Integer, File> fileMap = new TreeMap<Integer, File>(disk.getFileMap());
		Set<Integer> fileNums = fileMap.keySet();

		System.out.print("Reading memory... ");
		for(Integer file: fileNums){
			int ID = file.intValue();
			int type = disk.getFileType(ID);
			int newID;
			switch(type){
			case MEM_DICT:
				BlockwiseDictionaryDiskBlock dicBlock = new BlockwiseDictionaryDiskBlock(fileMap.get(file));
				newID = getBlockArrayIndex(ID);
				if(newID == -1) newID = allocateBlock(nextID++, MEM_DICT, 0);
				dicBlock.toMemory((BlockwiseDictionaryBlock)blocks[newID]);
				break;

			case MEM_SKIPPTR:
				SkipPointerDiskBlock skipBlock = new SkipPointerDiskBlock(fileMap.get(file));
				newID = getBlockArrayIndex(ID);
				if(newID == -1) newID = allocateBlock(nextID++, MEM_SKIPPTR, 0);
				skipBlock.toMemory((SkipPointerBlock)blocks[newID]);
				break;

			case MEM_INVINDEX:
				BlockwiseInvertIndexDiskBlock invBlock = new BlockwiseInvertIndexDiskBlock(fileMap.get(file));
				newID = getBlockArrayIndex(ID);
				if(newID == -1) newID = allocateBlock(nextID++, MEM_INVINDEX, 0);
				invBlock.toMemory((BlockwiseInvertIndexBlock)blocks[newID]);
				break;

			case MEM_CONT:
				BlockwiseContentDiskBlock contBlock = new BlockwiseContentDiskBlock(fileMap.get(file));
				newID = getBlockArrayIndex(ID);
				if(newID == -1) newID = allocateBlock(nextID++, MEM_CONT, 0);
				contBlock.toMemory((BlockwiseContentBlock)blocks[newID]);
				break;

			case MEM_PURECONT:
				AppendixDiskBlock pureBlock = new AppendixDiskBlock(fileMap.get(file));
				newID = getBlockArrayIndex(ID);
				if(newID == -1) newID = allocateBlock(nextID++, MEM_PURECONT, 0);
				pureBlock.toMemory((BlockwiseContentBlock)blocks[newID]);
				break;

			default:
				break;
			}
		}

		// 読み込んだブロックは消去
		for(int i = 0; i < blocks.length; i++){
			if(blocks[i] != null){
				disk.dropFile(blocks[i].getID());
			}
		}

		disk.reloadFileList();

		System.out.println("Done.");
	}

	public void saveMemory(int index){
		if(blocks[index] != null){
			// フラグを立てておく
			blocks[index].setAboutToFlush(true);
			// 既存のファイルがあれば消去
			disk.dropFile(blocks[index].getID());
			switch(blocks[index].getType()){
			case MEM_DICT:
				BlockwiseDictionaryDiskBlock dicBlock = new BlockwiseDictionaryDiskBlock(
						(BlockwiseDictionaryBlock)blocks[index], dir);
				dicBlock.save(0, blockSize, dir);
				break;

			case MEM_SKIPPTR:
				SkipPointerDiskBlock skipBlock = new SkipPointerDiskBlock((SkipPointerBlock)blocks[index],
						dir);
				skipBlock.save(0, blockSize, dir);
				break;

			case MEM_INVINDEX:
				BlockwiseInvertIndexDiskBlock invBlock = new BlockwiseInvertIndexDiskBlock(
						(BlockwiseInvertIndexBlock)blocks[index], dir);
				invBlock.save(0, blockSize, dir);
				break;

			case MEM_CONT:
				BlockwiseContentDiskBlock contBlock = new BlockwiseContentDiskBlock(
						(BlockwiseContentBlock)blocks[index], dir);
				contBlock.save(0, blockSize, dir);
				break;

			case MEM_PURECONT:
				AppendixDiskBlock pureBlock = new AppendixDiskBlock((AppendixBlock)blocks[index], dir);
				pureBlock.save(0, blockSize, dir);
				break;

			default:
				break;
			}
			// ブロック消去
			blocks[index] = null;
			disk.reloadFileList();
		}
	}

	// ブロック取り出し
	public byte[] leftBlock(int blockIndex) throws Exception{
		int casetteNum = getBlockArrayIndex(blockIndex);
		if(casetteNum != -1){
			// メモリ
			return blocks[casetteNum].getData();
		}else if(isLocal.get(blockIndex)){
			// ディスク
			return disk.getDiskblock_Blockwise(blockIndex).getData();
		}

		// ないよ
		return null;
	}

	public void clearBlock(int blockIndex){
		int casetteNum = getBlockArrayIndex(blockIndex);
		if(casetteNum != -1){
			// メモリ
			blocks[casetteNum] = null;
		}else if(isLocal.get(blockIndex)){
			// ディスク
			File fileBlock = disk.getDescriptor(blockIndex);
			fileBlock.delete();
		}
		isLocal.clear(blockIndex);
	}

	public void joinBlock(byte[] blockdata, int blockIndex, int blockType, int contentType)
			throws DataMatchingException, IOException{
		int casetteNum = getBlockArrayIndex(blockIndex);
		if(casetteNum != -1){
			// なにかが おかしい
			throw new DataMatchingException();
		}

		casetteNum = allocateBlock(blockIndex, blockType, contentType);

		blocks[casetteNum].setData(blockdata);
	}

	public int getBlockType(int blockIndex) throws Exception{
		int blockInMem = getBlockArrayIndex(blockIndex);
		if(blockInMem != -1) return blocks[blockInMem].getType();
		else if(isLocal.get(blockIndex)) return disk.getFileType(blockIndex);
		else return -1;
	}

	public int getContentType(int blockIndex){
		int blockInMem = getBlockArrayIndex(blockIndex);
		if(blockInMem != -1){
			int type = blocks[blockInMem].getType();
			if(type == MEM_CONT) return ((BlockwiseContentBlock)blocks[blockInMem]).getContentIndex();
			else if(type == MEM_PURECONT) return ((AppendixBlock)blocks[blockInMem]).getSupplementIndex();
		}else if(isLocal.get(blockIndex)){
			try{
				int type = disk.getFileType(blockIndex);
				if(type == MEM_CONT) return ((BlockwiseContentDiskBlock)disk
						.getDiskblock_Blockwise(blockIndex)).getContentIndex();
				else if(type == MEM_PURECONT)
					return ((AppendixDiskBlock)disk.getDiskblock_Blockwise(blockIndex)).getSupplementIndex();
			}catch(Exception e){}
		}

		return -1;
	}

	public Integer[] getBlockTypeList(){
		return blockTypes.toArray(new Integer[0]);
	}

	public void saveMemories(){
		// 今までの記録を保存
		System.out.print("Writing to memory... ");
		for(int i = 0; i < blocks.length; i++){
			saveMemory(i);
		}

		System.out.println("Done.");
	}

	public int getMinimumBlock(int type){
		int minimum = Integer.MAX_VALUE;
		int minimumArray = -1;
		for(int i = 0; i < blocks.length; i++){
			if(blocks[i] != null && blocks[i].getType() == type && blocks[i].getID() < minimum){
				minimum = blocks[i].getID();
				minimumArray = i;
			}
		}

		return minimumArray;
	}

	// デバッグ用
	public void showMemoryElapsed(){
		System.out.println("--  Disk  --");
		Set<Integer> keys = disk.getFileMap().keySet();
		for(Integer diskID: keys){
			DiskBlock diskBlock;
			try{
				File desc = disk.getFileMap().get(diskID);
				diskBlock = new DiskBlock(desc){
					@Override
					public int toMemory(MemoryBlock memBlock){
						throw new NoSuchMethodError("Unimplemented method");
					}

					@Override
					public byte[] getData(){
						throw new NoSuchMethodError("Unimplemented method");
					}
				};
				System.out.println(diskID + ": " + desc.length() + " " + diskBlock.getTypeStr());
			}catch(Exception e){}
		}

		System.out.println("-- Memory --");
		for(int i = 0; i < blocks.length; i++){
			if(blocks[i] != null)
				System.out.println(blocks[i].getID() + ": " + blocks[i].getUsed() + " "
						+ blocks[i].getClassName());
		}
		System.out.println("-- Cloud  --");
		for(int i = 0; i < blockTypes.size(); i++){
			System.out.println(i + ": " + blockTypes.get(i));
		}
	}

	// デバッグ用
	public void showMemoryType(){
		for(int i = 0; i < blocks.length; i++)
			if(blocks[i] != null) System.out.println(i + ": " + blocks[i].getClassName());
	}

	public void setBlockSize(int blockSize){
		this.blockSize = blockSize;
	}

	public void setBlockNum(int blockNum){
		this.blockNum = blockNum;
	}

	public int getBlockSize(){
		return blockSize;
	}

	public int getBlockNum(){
		return blockNum;
	}

	public int getContentCount() throws Exception{
		int result = 0;
		// メモリ
		for(int i = 0; i < blocks.length; i++){
			if(blocks[i] != null && blocks[i] instanceof BlockwiseContentBlock){
				result += ((BlockwiseContentBlock)blocks[i]).getCount();
			}
		}

		// ディスク
		Collection<File> files = disk.getFileMap().values();
		for(File file: files){
			BlockwiseContentDiskBlock blk = new BlockwiseContentDiskBlock(file);

			if(blk.getType() == MEM_CONT){
				result += blk.getCount();
			}
		}

		return result;
	}

	public int getTermCount() throws Exception{
		int result = 0;
		for(int i = 0; i < blocks.length; i++){
			if(blocks[i] != null && blocks[i] instanceof BlockwiseDictionaryBlock){
				int count = ((BlockwiseDictionaryBlock)blocks[i]).getTermCount();
				result += count;
			}
		}

		// ディスク
		Collection<File> files = disk.getFileMap().values();
		for(File file: files){
			BlockwiseDictionaryDiskBlock blk = new BlockwiseDictionaryDiskBlock(file);

			if(blk.getType() == MEM_DICT){
				result += blk.getTermCount();
			}
		}

		return result;
	}

	// 外部からブロック情報だけ受け取る
	public void registerBlock(int blockIndex, int blockType, int contentIndex){
		if(nextID <= blockIndex) nextID = blockIndex + 1;
		switch(blockType){
		case MEM_DICT:
			if(firstBlock[MEM_DICT] == -1) firstBlock[MEM_DICT] = blockIndex;
			if(lastBlock[MEM_DICT] < blockIndex) lastBlock[MEM_DICT] = blockIndex;
			break;
		case MEM_INVINDEX:
			if(firstBlock[MEM_INVINDEX] == -1) firstBlock[MEM_INVINDEX] = blockIndex;
			if(lastBlock[MEM_INVINDEX] < blockIndex) lastBlock[MEM_INVINDEX] = blockIndex;
			break;
		case MEM_SKIPPTR:
			if(firstBlock[MEM_SKIPPTR] == -1) firstBlock[MEM_SKIPPTR] = blockIndex;
			if(lastBlock[MEM_SKIPPTR] < blockIndex) lastBlock[MEM_SKIPPTR] = blockIndex;
			break;
		case MEM_CONT:
			if(contentFirst[contentIndex] == -1) contentFirst[contentIndex] = blockIndex;
			if(contentLast[contentIndex] < blockIndex) contentLast[contentIndex] = blockIndex;
			break;
		case MEM_PURECONT:
			if(supplementFirst[contentIndex] == -1) supplementFirst[contentIndex] = blockIndex;
			if(supplementLast[contentIndex] < blockIndex) supplementLast[contentIndex] = blockIndex;
			break;
		default:
			break;
		}
		blockTypes.set(blockIndex, Integer.valueOf(100 * contentIndex + blockType));
	}

	public int getBlockAmount(){
		return nextID;
	}

	public boolean existsLocal(int blockIndex){
		return isLocal.get(blockIndex);
	}

	public void pushBlock(int blockIndex, int blockType, int contentIndex, boolean isAllocate){
		if(isAllocate) allocateBlock(blockIndex, blockType, contentIndex);
		else registerBlock(blockIndex, blockType, contentIndex);
	}

	public class FuncCaller implements Callable<byte[]>{
		private String func;
		private int blockIndex;
		private byte[] argbytes;
		private boolean atOnce;
		private boolean isGet;

		public FuncCaller(String func, int blockIndex, byte[] argbytes, boolean atOnce, boolean isGet){
			this.func = func;
			this.blockIndex = blockIndex;
			this.argbytes = argbytes;
			this.atOnce = atOnce;
			this.isGet = isGet;
		}

		@Override
		public byte[] call() throws Exception{
			if(isGet) return memman.callGetFunc(func, blockIndex, argbytes, atOnce);
			else return memman.callSetFunc(func, blockIndex, argbytes, atOnce);
		}
	}

}
